from collections import Counter, defaultdict
import gzip, csv, json, math, hashlib


def get_cosine(vec1, vec2):
	intersection = set(vec1.keys()) & set(vec2.keys())
	numerator = sum([vec1[x] * vec2[x] for x in intersection])
	sum1 = sum([vec1[x]**2 for x in vec1.keys()])
	sum2 = sum([vec2[x]**2 for x in vec2.keys()])
	denominator = math.sqrt(sum1) * math.sqrt(sum2)
	if not denominator:
		return 0.0
	else:
		return float(numerator) / denominator


def merge(vec1, vec2):
	result = Counter()
	for term in set(vec1.keys() + vec2.keys()):
		result[term] = float(vec1[term] + vec2[term]) / 2
	return result


BOOKS = json.loads(open("bookids.json",'r').readline())
USERS = json.loads(open("userids.json",'r').readline())
NBOOKS = []

iterator = 0
DOCS = []


temp = list()
for i in xrange(0,304):
	temp.append(list())
	for k in xrange(0,110):
		temp[i].append(0)

ff = open("scorevectors.csv",'r')
csvreader = csv.reader(ff, delimiter='\t', quotechar='"')
"""
for iterator in xrange(0,20):
	NBOOKS.append(csvreader.next())

"""
b = 0
for wlist in csvreader:
	for i in xrange(0,109):
		temp[b][i] = float(wlist[i])
	b += 1

ff.close()


for i in xrange(0,303):
	Vector = Counter()
	for k in xrange(0,109):
		Vector[BOOKS[k]] = temp[i][k]
	Vector += Counter()
	DOCS.append((USERS[i], Vector))
	





CLUSTERS = defaultdict(list)
CENTROIDS = defaultdict(Counter)

CLUSTERS[hashlib.md5(DOCS[0][0]).hexdigest()].append(DOCS[0][0])
CENTROIDS[hashlib.md5(DOCS[0][0]).hexdigest()] = DOCS[0][1]
		
for user in DOCS[1:]:	# iterate through the book collection
	temp = Counter()
	for centroid in CENTROIDS:	# check book against all clusters
		temp[centroid]  = get_cosine(CENTROIDS[centroid],user[1])
	(c,m) = temp.most_common(1)[0]	# find the cluster with the maximum similarity
	if m > 0.9 :			# if similarity is more than 0.9 add book to the cluster
		CLUSTERS[c].append(user[0])
		CENTROIDS[c] = merge(CENTROIDS[c],user[1])
	else:				# else create new cluster
		CLUSTERS[hashlib.md5(user[0]).hexdigest()].append(user[0])
		CENTROIDS[hashlib.md5(user[0]).hexdigest()] = user[1]
		

BOOKID_CLUSTER = dict()
for clust in CLUSTERS:
  for user in CLUSTERS[clust]:
     BOOKID_CLUSTER[user] = clust


keys = CENTROIDS.keys()
outer = []
for vec1 in keys:
  inner = []
  for vec2 in keys:
     inner.append(get_cosine(CENTROIDS[vec1],CENTROIDS[vec2]))
  outer.append(inner)


[len(CLUSTERS[c]) for c in CLUSTERS]



